package com.example.hadoop.mapreduce.inverindex;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

/**
 * Created with IntelliJ IDEA.
 *
 * @Auther: Brian
 * @Date: 2020/04/28/9:36
 * @Description:
 * 将每个文件中出现的单词索引到一个文件中
 * a.txt:
 *      hello tom
 *      hello jerry
 *      hello tom
 *
 * b.txt
 *      hello jerry
 *      hello jerry
 *      tom jerry
 *
 * 输出：
 * hello a.txt-->3  b.txt-->2 c.txt-->1
 * jerry a.txt-->2  b.txt-->2 c.txt-->1
 * ......
 *
 * 1、将每个文件中出现的单词和对应文件名进行统计
 *  hello--a.txt-->3
 *  tom--a.txt-->2
 *  ......
 *
 * 2、将第一步输出结果再按单词作为key，其他信息作为values拼接起来
 */
public class InverIndexStepOne {

    static class InverIndexStepOneMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
        Text k = new Text();
        IntWritable v = new IntWritable(1);
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String line = value.toString();
            FileSplit inputSplit = (FileSplit) context.getInputSplit();
            String fileName = inputSplit.getPath().getName();
            String[] words = line.split(" ");
            for (String word : words) {
                k.set(word + "#" + fileName);
                context.write(k, v);
            }
        }
    }

    /**
     *     输入：
     *     <hello#a.txt,1><hello#a.txt,1>...
     *  *  <tom#a.txt,1><tom#a.txt,1>...
     */
    static class InverIndexStepOneReducer extends Reducer<Text, IntWritable, Text, NullWritable> {

        @Override
        protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
            int count = 0;
            for (IntWritable writable : values) {
                 count += writable.get();
            }
            key.set(key.toString() + "-->" + count);
            context.write(key, NullWritable.get());
        }
    }

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        if (args == null || args.length < 2) {
            args[0] = "/wordcount/flowinput";
            args[1] = "/wordcount/flowoutput";
        }

        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        //设置jar包路径
        job.setJarByClass(InverIndexStepOne.class);

        //设置Map Reduce类
        job.setMapperClass(InverIndexStepOneMapper.class);
        job.setReducerClass(InverIndexStepOneReducer.class);

        //设置Map输出
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        //设置最终输出
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);

        //设置输入输出路径
        FileInputFormat.setInputPaths(job, new Path(args[0]));
        Path outputPath = new Path(args[1]);
        FileSystem fileSystem = FileSystem.get(conf);
        if (fileSystem.exists(outputPath)) {
            fileSystem.delete(outputPath, true);
        }
        FileOutputFormat.setOutputPath(job, outputPath);

        boolean successful = job.waitForCompletion(true);
        System.out.println("Successfully? --> " + successful);
        System.exit(successful ? 0 : 1);
    }
}
